{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "GXlFwZFG03LB",
    "outputId": "9ff10d1d-1441-4bc3-fb6b-7a2dfcc1e7db"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x782a68ef4030>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import DataLoader\n",
    "from datasets import load_dataset\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "torch.manual_seed(12046)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "G37XHrLc03LF"
   },
   "outputs": [],
   "source": [
    "# 一些超参数\n",
    "learning_rate = 1e-3\n",
    "eval_iters = 10\n",
    "batch_size=1000\n",
    "sequence_len=64\n",
    "# 如果有GPU，该脚本将使用GPU进行计算\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "53cFFO3C03LF"
   },
   "outputs": [],
   "source": [
    "raw_datasets = load_dataset(\"code_search_net\", \"python\")\n",
    "datasets = raw_datasets['train'].filter(lambda x: 'apache/spark' in x['repository_name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "fqu8u80A03LG",
    "outputId": "e4b19214-fc74-4bea-8ac3-05aca4c23dd3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "98"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class char_tokenizer:\n",
    "\n",
    "    def __init__(self, data):\n",
    "        # 数据中出现的所有字符构成字典\n",
    "        chars = sorted(list(set(''.join(data))))\n",
    "        # 预留一个位置给结尾的特殊字符\n",
    "        self.char2ind = {s : i + 1 for i, s in enumerate(chars)}\n",
    "        self.char2ind['<|e|>'] = 0\n",
    "        self.ind2char = {i : s for s, i in self.char2ind.items()}\n",
    "\n",
    "    def encode(self, text):\n",
    "        return [self.char2ind[c] for c in text]\n",
    "\n",
    "    def decode(self, enc):\n",
    "        if isinstance(enc, int):\n",
    "            return self.ind2char[enc]\n",
    "        return [self.ind2char[i] for i in enc]\n",
    "\n",
    "tok = char_tokenizer(datasets['whole_func_string'])\n",
    "len(tok.char2ind)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "lEXNR5-S03LH"
   },
   "outputs": [],
   "source": [
    "class RNN(nn.Module):\n",
    "\n",
    "    def __init__(self, input_size, hidden_size):\n",
    "        '''\n",
    "        循环神经网络的神经元（支持批量计算）\n",
    "        参数\n",
    "        ----\n",
    "        input_size ：int，输入数据的特征长度\n",
    "        hidden_size ：int，隐藏状态的特征长度\n",
    "        '''\n",
    "        super().__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.i2h = nn.Linear(input_size + hidden_size, hidden_size)\n",
    "\n",
    "    def forward(self, x, hidden=None):\n",
    "        '''\n",
    "        向前传播\n",
    "        参数\n",
    "        ----\n",
    "        x ：torch.FloatTensor\n",
    "            输入数据的集合，形状为(B, T, C)，其中B表示批量大小，T表示文本长度，C表示文字特征的长度（input_size）\n",
    "        hidden ：torch.FloatTensor\n",
    "            初始的隐藏状态，形状为(B, H)，其中H表示隐藏状态的长度（hidden_size）\n",
    "        返回\n",
    "        ----\n",
    "        hidden ：torch.FloatTensor，所有隐藏状态的集合，形状为(B, T, H)\n",
    "        '''\n",
    "        re = []\n",
    "        B, T, C = x.shape\n",
    "        x = x.transpose(0, 1)  # (T, B, C)\n",
    "        if hidden is None:\n",
    "            hidden = self.init_hidden(B, x.device)\n",
    "        for i in range(T):\n",
    "            # x[i]的形状是(B, C); hidden的形状是(B, H)\n",
    "            combined = torch.cat((x[i], hidden), dim=1)  # (B, C + H)\n",
    "            hidden = F.relu(self.i2h(combined))  # (   B, H)\n",
    "            re.append(hidden)\n",
    "        result_tensor = torch.stack(re, dim=0)   # (T, B, H)\n",
    "        return result_tensor.transpose(0, 1)     # (B, T, H)\n",
    "\n",
    "    def init_hidden(self, B, device):\n",
    "        # 默认的初始隐藏状态全部等于0\n",
    "        return torch.zeros((B, self.hidden_size), device=device)\n",
    "\n",
    "class CharRNNBatch(nn.Module):\n",
    "\n",
    "    def __init__(self, vs):\n",
    "        '''\n",
    "        双层的循环神经网络（支持批量计算）\n",
    "        参数\n",
    "        ----\n",
    "        vs ：int，字典大小\n",
    "        '''\n",
    "        super().__init__()\n",
    "        # 定义文字嵌入的特征长度\n",
    "        self.emb_size = 256\n",
    "        # 定义隐藏状态的特征长度\n",
    "        self.hidden_size = 128\n",
    "        # 文字嵌入层\n",
    "        self.embedding = nn.Embedding(vs, self.emb_size)\n",
    "        # 随机失活\n",
    "        self.dp = nn.Dropout(0.4)\n",
    "        # 第一层循环神经网络\n",
    "        self.rnn1 = RNN(self.emb_size, self.hidden_size)\n",
    "        # 第二层循环神经网络\n",
    "        self.rnn2 = RNN(self.hidden_size, self.hidden_size)\n",
    "        # 语言建模头，根据隐藏状态预测下一个字母是什么\n",
    "        self.h2o = nn.Linear(self.hidden_size, vs)\n",
    "\n",
    "    def forward(self, x):\n",
    "        '''\n",
    "        向前传播\n",
    "        参数\n",
    "        ----\n",
    "        x ：torch.LongTensor，当前字母在字典中的位置，形状为(B, T)\n",
    "        返回\n",
    "        ----\n",
    "        output ：torch.FloatTensor，预测结果的logits，形状为(B, T, vs)\n",
    "        '''\n",
    "        emb = self.embedding(x)      # (B, T,  C)\n",
    "        h = self.dp(self.rnn1(emb))  # (B, T,  H)\n",
    "        # 第一层的隐藏状态是第二层的输入\n",
    "        h = self.dp(self.rnn2(h))    # (B, T,  H)\n",
    "        # 使用第二层的隐藏状态预测下一个字母是什么\n",
    "        output = self.h2o(h)         # (B, T, vs)\n",
    "        return output\n",
    "\n",
    "model = CharRNNBatch(len(tok.char2ind)).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "vgDVyHDL03LI",
    "outputId": "f9d86d52-3f06-4503-e3e5-08985129b4c1"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CharRNNBatch(\n",
       "  (embedding): Embedding(98, 256)\n",
       "  (dp): Dropout(p=0.4, inplace=False)\n",
       "  (rnn1): RNN(\n",
       "    (i2h): Linear(in_features=384, out_features=128, bias=True)\n",
       "  )\n",
       "  (rnn2): RNN(\n",
       "    (i2h): Linear(in_features=256, out_features=128, bias=True)\n",
       "  )\n",
       "  (h2o): Linear(in_features=128, out_features=98, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 展示模型结构\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "KZiIPgh603LI"
   },
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def generate_batch(model, idx, max_new_tokens=300):\n",
    "    '''\n",
    "    利用模型生成文本（反复使用模型进行预测）\n",
    "    参数\n",
    "    ----\n",
    "    model ：CharRNNBatch，生成文本的模型\n",
    "    idx ：torch.LongTensor，当前字母在字典中的位置，形状为(1, T)\n",
    "    max_new_tokens ：int，生成文本的最大长度\n",
    "    返回\n",
    "    ----\n",
    "    out ：list[int]，生成的文本\n",
    "    '''\n",
    "    # 将模型切换至评估模式\n",
    "    model.eval()\n",
    "    for _ in range(max_new_tokens):\n",
    "        # 限制背景长度，使之与模型训练时的状况更相符\n",
    "        # 当然也可以不限制\n",
    "        context = idx[:, -sequence_len:]\n",
    "        # 在文本生成时，模型的计算效率很低，因为有很多重复计算\n",
    "        logits = model(context)\n",
    "        # 只使用最后一个预测结果\n",
    "        logits = logits[:, -1, :]\n",
    "        probs = F.softmax(logits, dim=-1)\n",
    "        # 根据模型预测的概率，得到最终的预测结果（下一个字母）\n",
    "        # 这一步运算有一定随机性\n",
    "        ix = torch.multinomial(probs, num_samples=1)\n",
    "        idx = torch.cat((idx, ix), dim=1)\n",
    "        if ix.item() == 0:\n",
    "            break\n",
    "    # 将模型切换至训练模式\n",
    "    model.train()\n",
    "    return idx.tolist()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Xc6EEzgL03LI",
    "outputId": "197a5c39-d194-4d55-f19e-cc9bf3a5301f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "def*ZO(F/of(\"YP{BZE G|uw=3:1'_$?Q9NN[{KQ=CK(AM:iKcaR;+Q3j<sAegWS$ö0Nx!qyT3\"yMm5Za)'W~5\\Wm&B\"({r\n",
      "cdtMh^DA11zc<|e|>\n"
     ]
    }
   ],
   "source": [
    "# 使用模型来生成文本\n",
    "begin_text = torch.tensor(tok.encode('def'), device=device).unsqueeze(0)\n",
    "print(''.join(tok.decode(generate_batch(model, begin_text))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Vu7WY3Pe03LJ",
    "outputId": "2e3c0195-ba98-4eff-b9a2-a053ebd3a6ba"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([605913, 64]), torch.Size([605913, 64]))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def process(data, sequence_len=sequence_len):\n",
    "    '''\n",
    "    根据文本生成训练数据\n",
    "    '''\n",
    "    # text是字符串列表\n",
    "    text = data['whole_func_string']\n",
    "    inputs, labels = [], []\n",
    "    for i in text:\n",
    "        enc = tok.encode(i)\n",
    "        # 0对应着文本结束\n",
    "        enc += [0]\n",
    "        # 将文本转换为多个训练数据\n",
    "        for i in range(len(enc) - sequence_len):\n",
    "            inputs.append(enc[i: i + sequence_len])\n",
    "            # 预测标签是下一个字母，因此只需要挪动一个位置即可\n",
    "            labels.append(enc[i + 1: i + 1 + sequence_len])\n",
    "    return {'inputs': inputs, 'labels': labels}\n",
    "\n",
    "# 将数据分为训练集和测试集\n",
    "tokenized = datasets.train_test_split(test_size=0.1, seed=1024, shuffle=True)\n",
    "# 将文本转换为训练数据，里面包含inputs和labels\n",
    "tokenized = tokenized.map(process, batched=True, remove_columns=datasets.column_names)\n",
    "tokenized.set_format(type='torch', device=device)\n",
    "\n",
    "tokenized['train']['inputs'].shape, tokenized['train']['labels'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "QUAlXU5803LJ",
    "outputId": "3b9e68a6-0eb4-4432-88e7-0cbfdce72c8c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'inputs': tensor([[16, 76, 85,  ..., 69, 67, 80],\n",
       "         [ 2,  2,  2,  ..., 71, 72, 71],\n",
       "         [89, 37, 81,  ..., 70, 40, 75],\n",
       "         ...,\n",
       "         [76, 71, 69,  ...,  2,  2,  2],\n",
       "         [ 2, 19,  1,  ..., 74, 67, 86],\n",
       "         [48, 81, 84,  ..., 78,  2, 70]], device='cuda:0'),\n",
       " 'labels': tensor([[76, 85, 81,  ..., 67, 80,  2],\n",
       "         [ 2,  2,  2,  ..., 72, 71, 84],\n",
       "         [37, 81, 79,  ..., 40, 75, 71],\n",
       "         ...,\n",
       "         [71, 69, 86,  ...,  2,  2,  2],\n",
       "         [19,  1,  2,  ..., 67, 86,  2],\n",
       "         [81, 84, 79,  ...,  2, 70, 75]], device='cuda:0')}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 构建数据读取器\n",
    "train_loader = DataLoader(tokenized['train'], batch_size=batch_size, shuffle=True)\n",
    "test_loader = DataLoader(tokenized['test'], batch_size=batch_size, shuffle=True)\n",
    "# 获取一个批量的数据\n",
    "next(iter(test_loader))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Wlb2hwJ503LK",
    "outputId": "8cb04947-2c64-49c5-cbb8-6b0a4a30bcfc"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'train': 4.572356224060059, 'test': 4.573554515838623}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def estimate_loss(model):\n",
    "    re = {}\n",
    "    # 将模型切换至评估模式\n",
    "    model.eval()\n",
    "    re['train'] = _loss(model, train_loader)\n",
    "    re['test'] = _loss(model, test_loader)\n",
    "    # 将模型切换至训练模式\n",
    "    model.train()\n",
    "    return re\n",
    "\n",
    "@torch.no_grad()\n",
    "def _loss(model, data_loader):\n",
    "    \"\"\"\n",
    "    计算模型在不同数据集下面的评估指标\n",
    "    \"\"\"\n",
    "    loss = []\n",
    "    data_iter= iter(data_loader)\n",
    "    # 随机使用多个批量数据来预估模型效果\n",
    "    for k in range(eval_iters):\n",
    "        data = next(data_iter, None)\n",
    "        if data is None:\n",
    "            data_iter = iter(data_loader)\n",
    "            data = next(data_iter, None)\n",
    "        inputs, labels = data['inputs'], data['labels']\n",
    "        logits = model(inputs)\n",
    "        # 根据cross_entropy的定义，需要对logits进行转置运算\n",
    "        # 具体细节请参考cross_entropy的官方文档\n",
    "        logits = logits.transpose(-2, -1)\n",
    "        loss.append(F.cross_entropy(logits, labels).item())\n",
    "    return torch.tensor(loss).mean().item()\n",
    "\n",
    "estimate_loss(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "gqyC2yaA03LK"
   },
   "outputs": [],
   "source": [
    "def train_rnn(model, optimizer, data_loader, epochs=10):\n",
    "    lossi = []\n",
    "    for epoch in range(epochs):\n",
    "        for i, data in enumerate(data_loader, 0):\n",
    "            inputs, labels = data['inputs'], data['labels']\n",
    "            optimizer.zero_grad()\n",
    "            logits = model(inputs)\n",
    "            # 根据cross_entropy的定义，需要对logits进行转置运算\n",
    "            # 具体细节请参考cross_entropy的官方文档\n",
    "            logits = logits.transpose(-2, -1)\n",
    "            loss = F.cross_entropy(logits, labels)\n",
    "            lossi.append(loss.item())\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "        # 评估模型，并输出结果\n",
    "        stats = estimate_loss(model)\n",
    "        train_loss = f'train loss {stats[\"train\"]:.4f}'\n",
    "        test_loss = f'test loss {stats[\"test\"]:.4f}'\n",
    "        print(f'epoch {epoch:>2}: {train_loss}, {test_loss}')\n",
    "    return lossi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "XHzkjkOd03LK",
    "outputId": "b2bac2a9-48f5-4ae6-d644-75655eaee19c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch  0: train loss 1.4990, test loss 1.6117\n",
      "epoch  1: train loss 1.3739, test loss 1.4928\n",
      "epoch  2: train loss 1.3112, test loss 1.4462\n",
      "epoch  3: train loss 1.2822, test loss 1.4168\n",
      "epoch  4: train loss 1.2611, test loss 1.3976\n",
      "epoch  5: train loss 1.2440, test loss 1.3893\n",
      "epoch  6: train loss 1.2383, test loss 1.3751\n",
      "epoch  7: train loss 1.2313, test loss 1.3710\n",
      "epoch  8: train loss 1.2095, test loss 1.3622\n",
      "epoch  9: train loss 1.2253, test loss 1.3593\n"
     ]
    }
   ],
   "source": [
    "l = train_rnn(model, optim.Adam(model.parameters(), lr=learning_rate), train_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 448
    },
    "id": "e94Xbj3003LL",
    "outputId": "743f41aa-a6d1-4a50-d003-1726c5497988"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x782918314880>]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGdCAYAAADAAnMpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA6TElEQVR4nO3de3yU9Z33//dMkpkcZ3IiJ3IABDmHo0DAUwVFZS10u9ZFeqNdtbcubLV317ul3a22to2/dbW1hxurVrFaitUKWgoiC4IiAQGDcpBwJiHkQBKSyXGSzFy/P5JMCCSQQDJXYF7Px2MeMddck/nM90GTd7/fz/W9LIZhGAIAADCJ1ewCAABAYCOMAAAAUxFGAACAqQgjAADAVIQRAABgKsIIAAAwFWEEAACYijACAABMFWx2Ad3h9Xp16tQpRUVFyWKxmF0OAADoBsMwVF1drZSUFFmtXc9/XBFh5NSpU0pLSzO7DAAAcAkKCgqUmpra5fNXRBiJioqS1PJhHA6HydUAAIDucLlcSktL8/0d78oVEUbalmYcDgdhBACAK8zFWixoYAUAAKYijAAAAFMRRgAAgKkIIwAAwFSEEQAAYCrCCAAAMBVhBAAAmIowAgAATEUYAQAApiKMAAAAUxFGAACAqQgjAADAVFfEjfL6yh+2HFNBRZ3+eUqaRiRxAz4AAMwQ0DMjq784pWVbjyu/vM7sUgAACFiXFUaefvppWSwWPfbYYxc876233tKIESMUGhqqsWPHas2aNZfztr3G2npLY8PkOgAACGSXHEZ27Nih3//+98rMzLzgeVu3btX8+fP1wAMPKDc3V/PmzdO8efO0d+/eS33rXmNtySIyDOIIAABmuaQwUlNTowULFuill15STEzMBc99/vnndfvtt+vxxx/XyJEj9dRTT2nixIn67W9/e0kF9yZL68yIlywCAIBpLimMLFq0SHPmzNGsWbMuem5OTs55582ePVs5OTldvsbtdsvlcnV49IXWiRF5mRkBAMA0Pb6aZsWKFfrss8+0Y8eObp1fXFysxMTEDscSExNVXFzc5Wuys7P1k5/8pKel9ZivZ4QsAgCAaXo0M1JQUKBHH31Uf/rTnxQaGtpXNWnJkiWqqqryPQoKCvrkfaytn56ZEQAAzNOjmZFdu3aptLRUEydO9B3zeDz66KOP9Nvf/lZut1tBQUEdXpOUlKSSkpIOx0pKSpSUlNTl+9jtdtnt9p6UdkmYGQEAwHw9mhmZOXOm9uzZo927d/sekydP1oIFC7R79+7zgogkZWVlacOGDR2OrV+/XllZWZdXeS9ob2AljQAAYJYezYxERUVpzJgxHY5FREQoLi7Od3zhwoUaOHCgsrOzJUmPPvqobrrpJj377LOaM2eOVqxYoZ07d+rFF1/spY9w6dobWE0tAwCAgNbrO7Dm5+erqKjI9/306dO1fPlyvfjiixo3bpzefvttrVq16rxQYwb2GQEAwHyXfW+aTZs2XfB7Sbr77rt19913X+5b9Tp6RgAAMF9A35uGnhEAAMwX4GGk5Ss9IwAAmCegw4ivZ4Rb5QEAYJoADyPcmwYAALMRRsTVNAAAmCmgw4ivZ4SpEQAATBPgYYRlGgAAzBbQYaS9gRUAAJglwMMIPSMAAJgtoMNI+z4jhBEAAMwS2GFE9IwAAGC2gA4j7TfKM7cOAAACWYCHEe5NAwCA2QI7jLR+ehpYAQAwT0CHEfYZAQDAfIEdRlq/skwDAIB5AjqMtO8zYnIhAAAEsAAPIy1f6RkBAMA8AR1G6BkBAMB8AR5GWr7SMwIAgHkCOoxYmRkBAMB0AR5GWr4a3LcXAADTBHgY4WoaAADMFtBhxNfAyjoNAACmCfAw0vKVLAIAgHkCOozQMwIAgPkCPIzQMwIAgNkCOoy0b3pGGgEAwCwBHUasbHoGAIDpAjqMWMSmZwAAmC2gw0j7jfLMrQMAgEAW2GHE2tbAShoBAMAsAR1GuFEeAADmC+wwQs8IAACmC+gwQs8IAADmC/AwQs8IAABmC+gwQs8IAADmC+gwYrXQMwIAgNkCOowwMwIAgPkCOoz4ekZMrgMAgEAW4GGk5SsNrAAAmCegw4jvrr1ekwsBACCABXgYaflKzwgAAOYJ6DDC1TQAAJgvwMNI23+RRgAAMEtAhxELMyMAAJguoMNI+zINaQQAALMEdBhpW6VhZgQAAPMEdBixtn569hkBAMA8gR1GfHftNbkQAAACWECHEQs9IwAAmC6gw4iVTc8AADBdQIcRi7i0FwAAs/UojCxdulSZmZlyOBxyOBzKysrS2rVruzx/2bJlslgsHR6hoaGXXXRv8W16RhgBAMA0wT05OTU1VU8//bSGDRsmwzD02muvae7cucrNzdXo0aM7fY3D4VBeXp7v+7Y+jf6AnhEAAMzXozBy1113dfj+5z//uZYuXapt27Z1GUYsFouSkpIuvcI+RM8IAADmu+SeEY/HoxUrVqi2tlZZWVldnldTU6OMjAylpaVp7ty52rdv30V/ttvtlsvl6vDoC2wHDwCA+XocRvbs2aPIyEjZ7XY9/PDDWrlypUaNGtXpucOHD9crr7yid999V2+88Ya8Xq+mT5+ukydPXvA9srOz5XQ6fY+0tLSeltktbTMjZBEAAMxjMXq4/WhjY6Py8/NVVVWlt99+Wy+//LI2b97cZSA5W1NTk0aOHKn58+frqaee6vI8t9stt9vt+97lciktLU1VVVVyOBw9KfeCPjxQqm8t26HMVKfeW3x9r/1cAADQ8vfb6XRe9O93j3pGJMlms2no0KGSpEmTJmnHjh16/vnn9fvf//6irw0JCdGECRN0+PDhC55nt9tlt9t7WlqPWegZAQDAdJe9z4jX6+0wi3EhHo9He/bsUXJy8uW+ba/w3bXXa3IhAAAEsB7NjCxZskR33HGH0tPTVV1dreXLl2vTpk1at26dJGnhwoUaOHCgsrOzJUk//elPNW3aNA0dOlSVlZV65plndOLECT344IO9/0kuATMjAACYr0dhpLS0VAsXLlRRUZGcTqcyMzO1bt063XrrrZKk/Px8Wa3tky1nzpzRQw89pOLiYsXExGjSpEnaunVrt/pL/MHaj/Y8AQAgUPW4gdUM3W2A6amtR8p070vbdW1ipD747k299nMBAED3/34H9L1prOwzAgCA6QI6jLQt0tAzAgCAeQI6jFjZ9QwAANMFdhjhahoAAEwX0GGEe9MAAGC+gA4j7Q2spBEAAMwS0GGkrYGVLAIAgHkCOoy0zYxcAVutAABw1QroMNK+Hby5dQAAEMgCOozQMwIAgPkCOowwMwIAgPkCOozQMwIAgPkCPIy0fCWKAABgnoAOIxZ6RgAAMF1AhxHfdvA0jQAAYJqADiMWX8+IyYUAABDAAjqM0DMCAID5AjyM0DMCAIDZAjqMtO8zQhgBAMAsAR1G2mdGTC4EAIAAFtBhpG1mhE3PAAAwT0CHEStX0wAAYLqADiP0jAAAYL6ADiP0jAAAYL6ADiOWs/6bvhEAAMwR0GGkbWZEom8EAACzEEZa0TcCAIA5AjqMWM769PSNAABgjoAOI8yMAABgvoAOIx0bWE0rAwCAgBbQYaRDAyv37gUAwBQBHUbOyiLy0DQCAIApAjqMhAS1f3zCCAAA5gjoMBJktcjaOjvS6PGaWwwAAAEqoMOI1D470uRhZgQAADMEfBixtYaRxmZmRgAAMEPAh5GQ4LaZEcIIAABmCPgwwswIAADmCvgwEhLc0sHKzAgAAOYgjNDACgCAqQI+jNiC6BkBAMBMAR9G2mZG2GcEAABzEEaCWnpGaGAFAMAchBGWaQAAMFXAhxEb+4wAAGAqwkjbzEgzV9MAAGCGgA8jNLACAGAuwgjLNAAAmIowwtU0AACYKuDDCJueAQBgroAPI+09IzSwAgBgBsIIMyMAAJgq4MOIb58RekYAADBFj8LI0qVLlZmZKYfDIYfDoaysLK1du/aCr3nrrbc0YsQIhYaGauzYsVqzZs1lFdzbbK0NrMyMAABgjh6FkdTUVD399NPatWuXdu7cqVtuuUVz587Vvn37Oj1/69atmj9/vh544AHl5uZq3rx5mjdvnvbu3dsrxfcGekYAADCXxTCMy/orHBsbq2eeeUYPPPDAec/dc889qq2t1erVq33Hpk2bpvHjx+uFF17o9nu4XC45nU5VVVXJ4XBcTrnneWHzET299oC+PjFVz35jXK/+bAAAAll3/35fcs+Ix+PRihUrVFtbq6ysrE7PycnJ0axZszocmz17tnJyci71bXsdDawAAJgruKcv2LNnj7KystTQ0KDIyEitXLlSo0aN6vTc4uJiJSYmdjiWmJio4uLiC76H2+2W2+32fe9yuXpaZrfRMwIAgLl6PDMyfPhw7d69W9u3b9cjjzyi++67T/v37+/VorKzs+V0On2PtLS0Xv35Z2NmBAAAc/U4jNhsNg0dOlSTJk1Sdna2xo0bp+eff77Tc5OSklRSUtLhWElJiZKSki74HkuWLFFVVZXvUVBQ0NMyu63t0l4aWAEAMMdl7zPi9Xo7LKmcLSsrSxs2bOhwbP369V32mLSx2+2+y4fbHn3FNzPCPiMAAJiiRz0jS5Ys0R133KH09HRVV1dr+fLl2rRpk9atWydJWrhwoQYOHKjs7GxJ0qOPPqqbbrpJzz77rObMmaMVK1Zo586devHFF3v/k1wilmkAADBXj8JIaWmpFi5cqKKiIjmdTmVmZmrdunW69dZbJUn5+fmyWtsnW6ZPn67ly5frP/7jP/TDH/5Qw4YN06pVqzRmzJje/RSXwRbcetdewggAAKboURj5wx/+cMHnN23adN6xu+++W3fffXePivKn0OAgSZK7iTACAIAZAv7eNKG2ljBS3+QxuRIAAAJTwIeRsJCWMFLXSBgBAMAMhJHWMNLAzAgAAKYgjJy1THOZt+kBAACXIODDSGjrzIjHa6iJjc8AAPC7gA8jbcs0Ek2sAACYIeDDSEiQRUHWlr1G6BsBAMD/Aj6MWCwW3+xIPVfUAADgdwEfRqT2vhGWaQAA8D/CiKRwNj4DAMA0hBGJZRoAAExEGNFZW8ITRgAA8DvCiKSwkJZhYJkGAAD/I4zorGUawggAAH5HGFH7lvDsMwIAgP8RRnTWpb30jAAA4HeEEbFMAwCAmQgjIowAAGAmwojaNz1rYJkGAAC/I4yofZ+ROsIIAAB+RxgRyzQAAJiJMKL2MMKlvQAA+B9hRO37jDAzAgCA/xFGxD4jAACYiTCis3tGvCZXAgBA4CGMiO3gAQAwE2FEZ82MsEwDAIDfEUbU3jNS19hsciUAAAQewojO2oGVnhEAAPyOMKL2ZZpGj1fNHgIJAAD+RBhRewOrJDU0E0YAAPAnwogke3D7MNDECgCAfxFGJFksFraEBwDAJISRVmwJDwCAOQgjrdhrBAAAcxBGWoWGtAwFMyMAAPgXYaRVuC1YEhufAQDgb4SRVpH2ljBS3UAYAQDAnwgjrRxhLWHERRgBAMCvCCOtokJDJEnVDU0mVwIAQGAhjLRytIYRVz0zIwAA+BNhpFVUaFvPCDMjAAD4E2GklSOsdWaEnhEAAPyKMNKKmREAAMxBGGnV3jNCGAEAwJ8II60coewzAgCAGQgjrdp7RpgZAQDAnwgjraKYGQEAwBSEkVZtPSN1jR41ebwmVwMAQOAgjLRyhIXIYmn578o6lmoAAPAXwkirIKtFMeE2SVJFbaPJ1QAAEDgII2eJjWgJI+U1bpMrAQAgcBBGzuILI8yMAADgN4SRs8RFsEwDAIC/9SiMZGdn67rrrlNUVJQSEhI0b9485eXlXfA1y5Ytk8Vi6fAIDQ29rKL7CjMjAAD4X4/CyObNm7Vo0SJt27ZN69evV1NTk2677TbV1tZe8HUOh0NFRUW+x4kTJy6r6L4SF2mXRM8IAAD+FNyTk99///0O3y9btkwJCQnatWuXbrzxxi5fZ7FYlJSUdGkV+hHLNAAA+N9l9YxUVVVJkmJjYy94Xk1NjTIyMpSWlqa5c+dq3759Fzzf7XbL5XJ1ePgDyzQAAPjfJYcRr9erxx57TDNmzNCYMWO6PG/48OF65ZVX9O677+qNN96Q1+vV9OnTdfLkyS5fk52dLafT6XukpaVdapk9wswIAAD+ZzEMw7iUFz7yyCNau3attmzZotTU1G6/rqmpSSNHjtT8+fP11FNPdXqO2+2W293et+FyuZSWlqaqqio5HI5LKbdb8oqrNftXHykmPES5P76tz94HAIBA4HK55HQ6L/r3u0c9I20WL16s1atX66OPPupREJGkkJAQTZgwQYcPH+7yHLvdLrvdfimlXZa2ZZrK+iZ5vIaCrBa/1wAAQKDp0TKNYRhavHixVq5cqY0bN2rw4ME9fkOPx6M9e/YoOTm5x6/tazHhLTfLMwzpTB1LNQAA+EOPwsiiRYv0xhtvaPny5YqKilJxcbGKi4tVX1/vO2fhwoVasmSJ7/uf/vSn+uCDD3T06FF99tln+uY3v6kTJ07owQcf7L1P0UuCg6yKbg0k9I0AAOAfPVqmWbp0qSTp5ptv7nD81Vdf1f333y9Jys/Pl9XannHOnDmjhx56SMXFxYqJidGkSZO0detWjRo16vIq7yNxETZV1jWpvKZRSjS7GgAArn49CiPd6XXdtGlTh+9/+ctf6pe//GWPijJTXIRdR07XqryWjc8AAPAH7k1zjlgu7wUAwK8II+eIjWwJI2U1hBEAAPyBMHKOAa33pzld3WByJQAABAbCyDmSnC13FC6uIowAAOAPhJFzJDlaw4iLBlYAAPyBMHKOxNYwUuJiZgQAAH8gjJyjbZmmorZR7maPydUAAHD1I4ycIyY8RLbglmEpZakGAIA+Rxg5h8ViUaKj5YqaYpZqAADoc4SRTgyMDpMknTxTZ3IlAABc/QgjnciIjZAknSgnjAAA0NcII51IjwuXRBgBAMAfCCOdGBTXNjNSa3IlAABc/QgjnchgZgQAAL8hjHSibZmmvLZRNe5mk6sBAODqRhjphCM0RLERLXfvZakGAIC+RRjpQnosSzUAAPgDYaQL9I0AAOAfhJEuZLReUZNfwTINAAB9iTDShYzWZZrjZcyMAADQlwgjXWhbpsmvIIwAANCXCCNdaFumOVVVL3ezx+RqAAC4ehFGuhAfaVO4LUiGIRVU1JtdDgAAVy3CSBcsFgtNrAAA+AFh5AJoYgUAoO8RRi5g8ICWmZHDp2tMrgQAgKsXYeQCRiU7JEn7T7lMrgQAgKsXYeQCRqW0hJEDxS55vIbJ1QAAcHUijFzAoLgIhYUEqaHJq2NlNLECANAXCCMXEGS1aERylCRpfxFLNQAA9AXCyEXQNwIAQN8ijFxEW98IMyMAAPQNwshFMDMCAEDfIoxcxIgkh6wWqazGrdLqBrPLAQDgqkMYuYgwW5CGDIiUJO1jdgQAgF5HGOkGlmoAAOg7hJFuoIkVAIC+QxjphraZkS+ZGQEAoNcRRrphZGsYOVZeq1p3s8nVAABwdSGMdMOAKLsSouwyDOlAcbXZ5QAAcFUhjHQTfSMAAPQNwkg3cUUNAAB9gzDSTcyMAADQNwgj3dQ2M3KgyKVmj9fkagAAuHoQRropIy5C4bYguZu9OlZWa3Y5AABcNQgj3RRktfhmR744WWVyNQAAXD0IIz0wPi1akrS7oNLUOgAAuJoQRnpgHGEEAIBeRxjpgQnp0ZJarqg5U9tobjEAAFwlCCM9kBoTrlHJDnm8htbuLTa7HAAArgqEkR766vgUSdLavUUmVwIAwNWBMNJDt4xIkCTtOF4hd7PH5GoAALjyEUZ6aFhCpOIjbWpo8mp3fqXZ5QAAcMXrURjJzs7Wddddp6ioKCUkJGjevHnKy8u76OveeustjRgxQqGhoRo7dqzWrFlzyQWbzWKxKOuaeEnS1iPlJlcDAMCVr0dhZPPmzVq0aJG2bdum9evXq6mpSbfddptqa7vekXTr1q2aP3++HnjgAeXm5mrevHmaN2+e9u7de9nFm2X6NXGSpBzCCAAAl81iGIZxqS8+ffq0EhIStHnzZt14442dnnPPPfeotrZWq1ev9h2bNm2axo8frxdeeKFb7+NyueR0OlVVVSWHw3Gp5faaE+W1uumZTQoJsujzJ25TuC3Y7JIAAOh3uvv3+7J6RqqqWrZFj42N7fKcnJwczZo1q8Ox2bNnKycnp8vXuN1uuVyuDo/+JD02XAOjw9TkMfTxoTKzywEA4Ip2yWHE6/Xqscce04wZMzRmzJguzysuLlZiYmKHY4mJiSou7nqfjuzsbDmdTt8jLS3tUsvsExaLRXeOTZIkvff5KZOrAQDgynbJYWTRokXau3evVqxY0Zv1SJKWLFmiqqoq36OgoKDX3+NyzR0/UJL0/t5i7TheYXI1AABcuS4pjCxevFirV6/Whx9+qNTU1Auem5SUpJKSkg7HSkpKlJSU1OVr7Ha7HA5Hh0d/M2agU3PHp8jjNfTL9QfNLgcAgCtWj8KIYRhavHixVq5cqY0bN2rw4MEXfU1WVpY2bNjQ4dj69euVlZXVs0r7oe/MHCZJ2nnijBqa2AANAIBL0aMwsmjRIr3xxhtavny5oqKiVFxcrOLiYtXX1/vOWbhwoZYsWeL7/tFHH9X777+vZ599VgcOHNCTTz6pnTt3avHixb33KUwyJD5CSY5QNTZ7tfP4GbPLAQDgitSjMLJ06VJVVVXp5ptvVnJysu/x5ptv+s7Jz89XUVH7fVumT5+u5cuX68UXX9S4ceP09ttva9WqVRdser1SWCwWzRjasgHaxgOlJlcDAMCV6bL2GfGX/rbPyNk+2Fesb7++S8nOUH3y/VtktVrMLgkAgH7BL/uMQLrx2gGKsAWpqKpBHx9mzxEAAHqKMHKZQkOC9M9T0iVJz36QpytgogkAgH6FMNILHrn5GoXbgvTFySp9sL/k4i8AAAA+hJFeEB9p17dmDJIkvbD5iLnFAABwhSGM9JL7pg9SsNWi3PxKLXnnC5ZrAADoJsJIL0mICtXsMS27yv750wJ9ll9pbkEAAFwhCCO96Bfzxiq49dLeXSe4Xw0AAN1BGOlFzvAQPT57uCTpo4Nc5gsAQHcQRnrZ5EGxkqQth8v06ifHTK4GAID+jzDSyyamR2ve+BRJ0k/+tl8f7Cs2uSIAAPo3wkgvs1gs+u+7xynJESpJ+vbru3T0dI3JVQEA0H8RRvpAcJBVS+4c4ft+5wnu6AsAQFcII31k7viB+vaNQyRJX5ysNLcYAAD6McJIH8pMdUqSNn5Zqk+PcakvAACdIYz0oawhcYqNsOlUVYO+8fscLXh5m3YXVJpdFgAA/QphpA/FRdr17qIZ+uq4lqtrPjlcrv/18na5GppMrgwAgP6DMNLH0mLD9ew3xmnakJb9R6rdzXrotZ3s0AoAQCvCiB+EBFm1/MFp+s38CZKk7ccq9PWlOcorrja5MgAAzEcY8ROr1aK7xqXo4Zuu8R1jh1YAAAgjfveDO0borYezJEkrcwtVUdtockUAAJiLMGKCyRkxGjvQKXezV5N+tl4/W71fXq9hdlkAAJiCMGICi8Xi2xDNMKSXtxzTjuM0tAIAAhNhxCT/kJmsKa13+JWkpZuP6C87C7jsFwAQcCyGYfT79QGXyyWn06mqqio5HA6zy+k1roYm/eHjY3p+wyHfsQFRdr32rSkalXL1fE4AQGDq7t9vZkZM5AgN0aMzh2lyRozv2Olqtx57M1dXQEYEAKBXEEZMZrVa9Pz8Cbp3arqWPzRVtmCrDpbUaN8pl+oam/XWzgLVupvNLhMAgD7DMk0/869/2qU1e4o7HPvHiQP13DfGm1MQAACXiGWaK9RDNwxRsNXS4dg7nxVq36kqkyoCAKBvEUb6mQnpMVr6zUmaNiRW1wyI8B1f8PJ2vbD5iBqbvSZWBwBA72OZpp+rqmvSvS9v075TLknS6BSHhiVE6vYxybp9TJLJ1QEA0DWWaa4SzvAQvbtohn7xtbGSpH2nXFq1+5S+++ZuFVbWm1wdAACXjzByBQgOsureqen6zi1DlRBllyTVN3m04KVtys0/o9dzjuuPOce5HBgAcEVimeYKdKysVgte2qZTVQ0djv/u3omak5lsUlUAAHTEMs1VbHB8hH5z7wTf98nOUEnSk3/bpyOna8wqCwCAS8LMyBXsk8NlskjKiI/QzGc3qaGp5Uqb2AibBsWF6/u3j9DUIXHmFgkACFjd/ftNGLlK7DtVpe//9QvtLXR1OJ4aE6bxadH63m3DNTg+ootXAwDQ+wgjAai+0aOco2WKi7Dryb/tU25+ZYfnhwyI0D9fl6avjhuopNalHQAA+gphJMBV1TVpycovtP1ohWrczXKftVnakPgI/eH+6xQaYpUtyKq4SLuJlQIArlaEEfjUupt1/6ufasfxM50+/8d/maIbrx3g56oAAFc7wgg6OF5Wq1t/uVlNHkNR9mBVn3Mn4ESHXY/NulZew1CKM0yJjlDVNTZr8qBYkyoGAFzpCCM4z/5TLkWFBssRFqLqhiZV1Dbqq7/95IKv+esjWZqUQSABAPRcd/9+B/uxJphsVEr7PwRnWIhSY8I1d3yKdh4/0+XW8t9983NNyohRsNWixbcMVUYcV+QAAHoXMyOQJO0trNKq3EL985R0hYZY9fGhMv33ujyV1zZ2OG/e+BT9++zh2nn8jLYcLtOP7xolR2iISVUDAPozlmlw2Y6ertFz6w9q/ymXTlbWq/GsK3LOdl9Whkpcbg2IsuuxWcMUEmwloAAACCPofZ8cLtODr+1UfZPnoudOGRSru8Yl658mpSnMFuSH6gAA/Q1hBH3i5Jk61bibVev26PkNh/TRwdOSpHGpTu0vcqnJc/4/pxuGxevrE1MVF2nT9UPj5W72qr7Ro5gIm7/LBwD4EWEEfnOqsl7JzlB9eqxC/2/TEd0xJknLth7XgeLq884dmexQUVW9ahqadffkNNmCLPqyqFppseH69o1DNDwpyoRPAADoC4QRmMowDM359RbtL2q5V05ne5t0JsIWpAeuH6zbRifpdLVbWdfE6ejpWl2bGKndBZXKTI2WLZibTQPAlYAwAtOV1bhVXNWgZGeonGEh+rKoWmv2FmlIfIR2Hj+jN3cWSJJuHj5AxVUNnc6kdOaJu0bp/umDZLFY+rJ8AMBlIoyg3zt6ukarcgv1wPVD5DEMbT5YqoraJj2z7oAamjq/cqfNtCGxiou0K2tInAbFRSgyNFhRocFKcoQq3BakTw6X69qkSCVEcUNAADALYQRXrMq6Rh05Xav1+0v0wuYjkqR/vfkaDUuM1P/5y+e60L/Y9NhwTR0cq7d2nZQk/dstQxUTbtNtoxMVF2FXiatBh0prdMOweIWGcJUPAPQlwgiueB5vy2zJ5EGxvn1LvF5D2Wu/1EsfH5MkxUfaVeNuuuhMSpDVIq9h+ILM6BSHHrn5Gh0prVWxq16OsBA1NHr0wf4SjUp26HcLJqrW3cwdjQHgMhBGcFVr9ngVHNTeyFpZ16itR8r13x/kqbHZq1kjE7XvVJVSosN0qKTG10jbExaLNDE9RlMHx8oRFqK0mHDFRth03aAY33u7GpoUFhKkkCCaagHgXIQRoFWzx6vtxyrk8RqaMjhWpS63/v2tz3W0rEZ1jR7VNV58E7ezDYmPUHR4iPKKq1Xb6FGKM1T/PCVdA6LsmnFNvAZE2ZVbcEaJjlBdMyBSklRe45YzLESv5ZzQuFRnj+6GXN/oYeM4AFekPgsjH330kZ555hnt2rVLRUVFWrlypebNm9fl+Zs2bdJXvvKV844XFRUpKSmpW+9JGEFfOnmmTv/nL5/rjjFJumHYAG34skS1jR5lxIbrTF2jXtlyTKeqGnr8c4OtFs2fkq7S6gat21fiO24Ptur1B6bqdLVbwxIj9ZcdBQoNCdL3brv2vCuEfr/5iLLXHtDv7p2oOZnJl/1ZAcCf+iyMrF27Vp988okmTZqkf/zHf+x2GMnLy+tQSEJCgqzW7k1tE0ZgpoYmj2rczTpRXqvocJv2Flapqr5JaTHhrbMglcrNP6NNeadVcc6NBXvq3qnpGhIfIVdDs/afqtL/fFnqe+7zH98mZ3iIquqatP1YuVwNzbp1VKIOFLmUmRqtMFuQiqrq9fGhMhVU1Om7s66V1dox3DR5vAq2WrgsGoBf+GWZxmKxdDuMnDlzRtHR0Zf0PoQRXAly88/o3/6cq5Nn6uUIDdb1w+L1yeFyjUp2KDUmTJX1TYoJD9Hh0hp9ll/Zq+89dqBTc8en6Gd//9J37Pqh8frmtHS56pu1anehBkTZtfngaY1LjdbL903WJ4fLNDg+QhlxEd1+H3ezR/ZglowAdE93/34H+6ug8ePHy+12a8yYMXryySc1Y8aMLs91u91yu92+712unjcfAv42IT1GW75/i6obmtTkMRTbxb13mj1e7TpxRinRYSqqalBUaLAe+uNOuVsbb2vdzdpTWKVjZbWKj7Tr6xMH6r3PT6norKWihCi7ztQ1+u4FtKewSnsKqzq8z5bDZdpyuOy899988LSG/WitJCktNkyv3Hed/vxpgb4scinCHqxJGTF6+KYhKnY1KCTIqh3HKrTtaLkSnaH6r/fzJLXs8/J/bx+hiekxvTJ2l6KitlEnyms1wcQaAPSOPp8ZycvL06ZNmzR58mS53W69/PLLev3117V9+3ZNnDix09c8+eST+slPfnLecWZGEEiq6ppktUpRoSHyeA3tLqjUU6v363S1W39+aJriIm36vKBShZX1+u2Hh3WivE6SFGkPVk0nW+/HRtjk8Rqqqm/qtRoHRocpwWGX15D+17QMzRyRoF/+z0F9eqxCN147QGXVbkWH25QSHarocJvuHJukcFuwjNbLrM9dRsovb7kRY1ykTYdLa5Q1JO68c4qq6uU1pH//y+fKOVquV++/TiFBVk0dEtvpVU2rcgtlD7bqjrH03AD+1m+WaTpz0003KT09Xa+//nqnz3c2M5KWlkYYAS7A3eyRLcgqi8Wiv+woUEp0mKwWqaS6QfPGD/T1iVTWNeqD/SX6xZovVVnXJItFumHYAO0trDqv58Vi0XmbzH1jcqq2Ha1QfkVdj2sMCbJoQlqMvix2KdwWpDljU2So5Q1Gpzj1s7/vV2Vde1iaNz5F37guTZmp0Yq0B6vU1aBbf/lRp4Hq324ZqmlD4pTkDNX6/SWqb/To5Jl6/fWzlg3w/mPOSD14w5Bu1WkYBn01QC/o12Hk8ccf15YtW5STk9Ot8+kZAXqfu9mj1Z8XaUJ6tIYMiJTXa6isxq33Pj+lM3WN+sbkNA2Isqu4qkEnz9TrdLVbd45NVpgtSK6GJq3fV6IT5bX69cbDnf58i0VyhIYo2Rna7fsOdSUkyKLM1GidKK9VWc2lNQnHR9q04Xs3680d+ZoxNF6O0BAlOOyyBVn1188KVVxVr5Agqw6V1mjdvmJ9fWKq/vMfRsnVGnz+9xu7NC7VqWEJUdpxvEJNHq8WTMvQdYNi1djsldcwfLv6GoahoqoGxYTbzrssO6+4Wi99fFSPzhymSHuwNh4o1Z1jk9XY7FVkaLCCrIQgXD36dRi59dZbFRUVpXfeeadb5xNGgP6runXjt5Jqt/649biGJkTq6xNTVe1uliM0WBaLRQUVdXKEhejLIpc2fFmiXSfOKCEqVDERNkWHh8jd5NW6fcUqrKxXZqpT905JV2pMuH694ZDyK+pU7Dr/0uogq0VjUhw6Xl53yUtPjtBg2YKtlxxwJGloQqROVdarsdmrmSMTZA8O0qfHKlTsalC4LUgP3TBE9hCrZlwTr0FxEbr+/9uoanezrhsUo2JXgwoq6jU6xaG84molRNn12KxrteN4he7MTNboFIccoSEqcTXIGRai6PD2PiRXQ5MOFFVrckaMymrd2nX8jL4yIkEbD5RqVW6hfva1Mb57M9W4m7Uqt1B3ZabIGd6ym3GzxyurxXLeMhjQm/osjNTU1Ojw4Zb/JzRhwgQ999xz+spXvqLY2Filp6dryZIlKiws1B//+EdJ0q9+9SsNHjxYo0ePVkNDg15++WX95je/0QcffKCZM2f26ocBcOXyeg01NHsUbju/r/5YWa0+L6hUaIhVUwfHyWMYsgdbFdV6m4BtR8v1YV6p/mFsiqobmrTlcJn2FFZpYnqMJqRH66ODZXrlk2MXfP8hAyJ09HSt73tbkFWNngvfZsDfJqRHKze/UkmOUNU1NsvV0Ky4CJtqG5vV0ORVhC1ItWdt4jcsIVLXDIjUnsIqFVbWa8iACP3qnvEqr23Uf67aK3uwVdcNitXHh8o0MjlKczKTdbKiXq/lHNf8Ken63m3D5W72qKymUcfLajX9mjhJ0jufFSrRESp3s0fHymq1YGqG8ivqVFrdoOf/55AevGGIbhmRoGCrRfVNHoWFBGlPYZU8hqEJadGyWCwyDENHTtcoydmynGgYUoS982sqGpo8Ol3tVlpseIfjbX++WFLrv/osjHS1idl9992nZcuW6f7779fx48e1adMmSdJ//dd/6cUXX1RhYaHCw8OVmZmpH//4x53+jMv9MADQmYYmjz49VqGq+ialRIeqxu3RhPRofXywTGU1boXbgvRPk1LlamjWA8t2aERylB68foi+LHJpzECnPthfIo/Xq4SoUBVU1Om6wbG6ZkCk/rKzQM+sy9PA6DAtuXOEDpbUKDTEqnGp0ZqQHq2lm47ozR0FKq12X7TGgdFhmn5NnO8mj2YLslo0dECk8kral9iCrRaF24Lkaji/QfpCzm6qHpYQqW/NGKz3Pi/UtqMVio2wyd3kUbg9WH//t+sVHW7T+/uKFWK1aHdBpQxJ6/eX6FhZrQbFhWtoQqRSosM0ZqBTnxwu04YvS7UwK0MP33yNmpq9ig636X+/vlO7Cyr18E3XaO74gRoQ1XKPKcMwtP1YhSLtwaqsa9KMoXGyWCx67oM8FVY26OdfG6M/f5qvwjP1eujGIUp0dLzrt2EYOlZWq7gIu9bsLdKUwbEaEh+hgyU1CgsJUml1g293ZcMwdLrarZW5hbp3arovOBdU1Ck0JMhX09WO7eABwA+2Hy1Xely4kp1hXZ7jamjSmi+KtO+US2mxYZo/JV2Pv/WFYiJs2n6sXEdP1+qFb07U7NFJ2nK4TDHhNoWGWFXf6NWz6/P0tQkDVVHbqNz8Sp2oqNNNw+J1TUKk7MFWFVTUKzk6VNuOluuNbfmSpNtHJ+n9fcW+9x8SHyF3s9cXCOzB1g4B6R8yk+XxGlq7t1jpseFyhAVrb2HvbqkQZLXI473wn5tIe7AcocGXtOPxhcRH2mS1WFRZ19RhtuvcmsJCglTf5PHVkuiwKy7Srh/dOVIR9iA9tfpLbT542nd+aIhVYwc6teP4Gd+xJ+8apQ0HSrW3sEpnWpuxR6c49LUJA/X2rpO+/qk5mcmKj7BpdIpTA6LscoaH6Ok1BzQ21anrh8Ur2RmqEUkOGYahE+V12nG8QodP1+hgcbWSnGGaMTRO1yZGyRkWoh+/u1eHSmq05M6RuunaATpT19iy7OewKyM2QmG2IJW6GrTp4GndOTZZrvomJUTZdbSsVlaLVFXfrAlp0X2yZEcYAYArwLGyWp0or9XNwxMu6+fkl9fpzl9/rHFpTr3xwFT99bNC/ftbn+vx2cO16CtDzzvf4zX05o4CDU+K0qSMlr1aqhuaFGkPVnlto/6666QGxUeoocmjjQdKNWVwrOIj7Qq2WhQcZFVesUsT0mN0bUKU3sk9qRuGDVB+Ra3qG71657OT2nCgZffg+6cP0jcmp2lQfLhOnqnXd9/crX2nXLIHW/XwTdfo3d2FOl7e+ZVZWUPiFBpilSMsRPOnpOtAkUt/31MkZ5hNGw+UyGu0LF3ll9ep/DJ3P+6Ppg6OldViUc7R8m6/JiwkSB6v0eMlxgVT0/Xzr43taYkXRRgBgABT3dAkW7DVt0tuUVW9khyhpvVUnHt3bUlqbPZq44FSTUiP7rAMsutEhZZuOqLrBsXqm9My5DUM39JGZz4+dFp//jRfj88eoSaPV3/ddVJlNY3662cnNTA6TD/72hi9suWYthwu06Kbh2p0ikPxUXaVVbv1k7/tV0iwRePTYlTnbla1u1nfvmGIDpXWSJLunZKutz87qU8Ol+no6RpfWAoJsui1b03RkAGR2nCgRD9auVfXJkbqrswUPbv+oK+2m4cP0JgUpz7YX6zKuibFhNt0usbtu3Q+3BbU4xt0XkhUaLCqe7h01pnnvjFO/zgxtRcqakcYAQAEFHezR6/nnNDMkYkaHB8hr9eQq6Gpw1VIl6LZ420NJIaGJkT5jte4mxVhC/KFvff3Fkmy6PYxXd8EtsnjVUiQVR6voYraRr2/t0gjkh2yB1s1OsWpnCPlOnmmTgOi7Fqzp1jXJETojjHJGhQX7vu+1t2sf3ohR4bREmz++C8tAemzE2dUcKZOwVaLUqLDdKC4Wuv2FeuLk1W6YVi87p2SrtyCSg2Oj9Anh8u0+osiX103Dx+gX8+fIMcFAuClIIwAAHCV2n/KpUSHXbERtkue+Wr2ePXmzgJdPzReqTHhfbLHTb+7Nw0AAOgdo1Iu//+YBwdZtWBqRi9Uc/nOv5EDAACAHxFGAACAqQgjAADAVIQRAABgKsIIAAAwFWEEAACYijACAABMRRgBAACmIowAAABTEUYAAICpCCMAAMBUhBEAAGAqwggAADDVFXHXXsMwJLXcihgAAFwZ2v5ut/0d78oVEUaqq6slSWlpaSZXAgAAeqq6ulpOp7PL5y3GxeJKP+D1enXq1ClFRUXJYrH02s91uVxKS0tTQUGBHA5Hr/3cqxFj1T2MU/cxVt3DOHUfY9U9/hwnwzBUXV2tlJQUWa1dd4ZcETMjVqtVqampffbzHQ4H/3C7ibHqHsap+xir7mGcuo+x6h5/jdOFZkTa0MAKAABMRRgBAACmCugwYrfb9cQTT8hut5tdSr/HWHUP49R9jFX3ME7dx1h1T38cpyuigRUAAFy9AnpmBAAAmI8wAgAATEUYAQAApiKMAAAAUwV0GPnd736nQYMGKTQ0VFOnTtWnn35qdkl+9dFHH+muu+5SSkqKLBaLVq1a1eF5wzD04x//WMnJyQoLC9OsWbN06NChDudUVFRowYIFcjgcio6O1gMPPKCamho/foq+l52dreuuu05RUVFKSEjQvHnzlJeX1+GchoYGLVq0SHFxcYqMjNTXv/51lZSUdDgnPz9fc+bMUXh4uBISEvT444+rubnZnx+lzy1dulSZmZm+zZSysrK0du1a3/OMU+eefvppWSwWPfbYY75jjFWLJ598UhaLpcNjxIgRvucZp3aFhYX65je/qbi4OIWFhWns2LHauXOn7/l+/TvdCFArVqwwbDab8corrxj79u0zHnroISM6OtooKSkxuzS/WbNmjfGjH/3IeOeddwxJxsqVKzs8//TTTxtOp9NYtWqV8fnnnxtf/epXjcGDBxv19fW+c26//XZj3LhxxrZt24yPP/7YGDp0qDF//nw/f5K+NXv2bOPVV1819u7da+zevdu48847jfT0dKOmpsZ3zsMPP2ykpaUZGzZsMHbu3GlMmzbNmD59uu/55uZmY8yYMcasWbOM3NxcY82aNUZ8fLyxZMkSMz5Sn3nvvfeMv//978bBgweNvLw844c//KEREhJi7N271zAMxqkzn376qTFo0CAjMzPTePTRR33HGasWTzzxhDF69GijqKjI9zh9+rTvecapRUVFhZGRkWHcf//9xvbt242jR48a69atMw4fPuw7pz//Tg/YMDJlyhRj0aJFvu89Ho+RkpJiZGdnm1iVec4NI16v10hKSjKeeeYZ37HKykrDbrcbf/7znw3DMIz9+/cbkowdO3b4zlm7dq1hsViMwsJCv9Xub6WlpYYkY/PmzYZhtIxLSEiI8dZbb/nO+fLLLw1JRk5OjmEYLcHParUaxcXFvnOWLl1qOBwOw+12+/cD+FlMTIzx8ssvM06dqK6uNoYNG2asX7/euOmmm3xhhLFq98QTTxjjxo3r9DnGqd33v/994/rrr+/y+f7+Oz0gl2kaGxu1a9cuzZo1y3fMarVq1qxZysnJMbGy/uPYsWMqLi7uMEZOp1NTp071jVFOTo6io6M1efJk3zmzZs2S1WrV9u3b/V6zv1RVVUmSYmNjJUm7du1SU1NTh7EaMWKE0tPTO4zV2LFjlZiY6Dtn9uzZcrlc2rdvnx+r9x+Px6MVK1aotrZWWVlZjFMnFi1apDlz5nQYE4l/U+c6dOiQUlJSNGTIEC1YsED5+fmSGKezvffee5o8ebLuvvtuJSQkaMKECXrppZd8z/f33+kBGUbKysrk8Xg6/OOUpMTERBUXF5tUVf/SNg4XGqPi4mIlJCR0eD44OFixsbFX7Th6vV499thjmjFjhsaMGSOpZRxsNpuio6M7nHvuWHU2lm3PXU327NmjyMhI2e12Pfzww1q5cqVGjRrFOJ1jxYoV+uyzz5SdnX3ec4xVu6lTp2rZsmV6//33tXTpUh07dkw33HCDqqurGaezHD16VEuXLtWwYcO0bt06PfLII/rOd76j1157TVL//51+Rdy1F+gvFi1apL1792rLli1ml9JvDR8+XLt371ZVVZXefvtt3Xfffdq8ebPZZfUrBQUFevTRR7V+/XqFhoaaXU6/dscdd/j+OzMzU1OnTlVGRob+8pe/KCwszMTK+hev16vJkyfrF7/4hSRpwoQJ2rt3r1544QXdd999Jld3cQE5MxIfH6+goKDzOq5LSkqUlJRkUlX9S9s4XGiMkpKSVFpa2uH55uZmVVRUXJXjuHjxYq1evVoffvihUlNTfceTkpLU2NioysrKDuefO1adjWXbc1cTm82moUOHatKkScrOzta4ceP0/PPPM05n2bVrl0pLSzVx4kQFBwcrODhYmzdv1q9//WsFBwcrMTGRsepCdHS0rr32Wh0+fJh/U2dJTk7WqFGjOhwbOXKkb0mrv/9OD8gwYrPZNGnSJG3YsMF3zOv1asOGDcrKyjKxsv5j8ODBSkpK6jBGLpdL27dv941RVlaWKisrtWvXLt85GzdulNfr1dSpU/1ec18xDEOLFy/WypUrtXHjRg0ePLjD85MmTVJISEiHscrLy1N+fn6HsdqzZ0+H/6GvX79eDofjvF8gVxuv1yu32804nWXmzJnas2ePdu/e7XtMnjxZCxYs8P03Y9W5mpoaHTlyRMnJyfybOsuMGTPO23Lg4MGDysjIkHQF/E7v0/bYfmzFihWG3W43li1bZuzfv9/49re/bURHR3fouL7aVVdXG7m5uUZubq4hyXjuueeM3Nxc48SJE4ZhtFwGFh0dbbz77rvGF198YcydO7fTy8AmTJhgbN++3diyZYsxbNiwq+7S3kceecRwOp3Gpk2bOlxeWFdX5zvn4YcfNtLT042NGzcaO3fuNLKysoysrCzf822XF952223G7t27jffff98YMGDAVXd54Q9+8ANj8+bNxrFjx4wvvvjC+MEPfmBYLBbjgw8+MAyDcbqQs6+mMQzGqs33vvc9Y9OmTcaxY8eMTz75xJg1a5YRHx9vlJaWGobBOLX59NNPjeDgYOPnP/+5cejQIeNPf/qTER4ebrzxxhu+c/rz7/SADSOGYRi/+c1vjPT0dMNmsxlTpkwxtm3bZnZJfvXhhx8aks573HfffYZhtFwK9p//+Z9GYmKiYbfbjZkzZxp5eXkdfkZ5ebkxf/58IzIy0nA4HMa3vvUto7q62oRP03c6GyNJxquvvuo7p76+3vjXf/1XIyYmxggPDze+9rWvGUVFRR1+zvHjx4077rjDCAsLM+Lj443vfe97RlNTk58/Td/6l3/5FyMjI8Ow2WzGgAEDjJkzZ/qCiGEwThdybhhhrFrcc889RnJysmGz2YyBAwca99xzT4e9Mxindn/729+MMWPGGHa73RgxYoTx4osvdni+P/9OtxiGYfTt3AsAAEDXArJnBAAA9B+EEQAAYCrCCAAAMBVhBAAAmIowAgAATEUYAQAApiKMAAAAUxFGAACAqQgjAADAVIQRAABgKsIIAAAwFWEEAACY6v8HZUB5kVZIS4cAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(torch.tensor(l).view(-1, 10).mean(1).numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ygy6T1JZ03LL",
    "outputId": "e8b3da29-1fd1-4e0e-8b61-7d56d420a8fc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "def name=2.9], False,\n",
      "                \"fiend num rawfo data a verifien the resunc to end\n",
      "                # Deraming or not 'las1: semrine DataFrame.\n",
      "\n",
      "        >>> c.ichurrert.gat.ines = MLsib, inmente`,\n",
      "               whineQuiter is not None in data.\n",
      "\n",
      "    2.e dlec('dingentrine())\n",
      "    __bosle' matrix lic\n"
     ]
    }
   ],
   "source": [
    "# 使用模型来生成文本\n",
    "begin_text = torch.tensor(tok.encode('def'), device=device).unsqueeze(0)\n",
    "print(''.join(tok.decode(generate_batch(model, begin_text))))"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "V100",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
